{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ch `10`: Concept `02`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Recurrent Neural Network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import the relevant libraries:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.contrib import rnn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the RNN model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SeriesPredictor:\n",
    "\n",
    "    def __init__(self, input_dim, seq_size, hidden_dim=10):\n",
    "        # Hyperparameters\n",
    "        self.input_dim = input_dim\n",
    "        self.seq_size = seq_size\n",
    "        self.hidden_dim = hidden_dim\n",
    "\n",
    "        # Weight variables and input placeholders\n",
    "        self.W_out = tf.Variable(tf.random_normal([hidden_dim, 1]), name='W_out')\n",
    "        self.b_out = tf.Variable(tf.random_normal([1]), name='b_out')\n",
    "        self.x = tf.placeholder(tf.float32, [None, seq_size, input_dim])\n",
    "        self.y = tf.placeholder(tf.float32, [None, seq_size])\n",
    "\n",
    "        # Cost optimizer\n",
    "        self.cost = tf.reduce_mean(tf.square(self.model() - self.y))\n",
    "        self.train_op = tf.train.AdamOptimizer().minimize(self.cost)\n",
    "\n",
    "        # Auxiliary ops\n",
    "        self.saver = tf.train.Saver()\n",
    "\n",
    "    def model(self):\n",
    "        \"\"\"\n",
    "        :param x: inputs of size [T, batch_size, input_size]\n",
    "        :param W: matrix of fully-connected output layer weights\n",
    "        :param b: vector of fully-connected output layer biases\n",
    "        \"\"\"\n",
    "        cell = rnn.BasicLSTMCell(self.hidden_dim, reuse=tf.get_variable_scope().reuse)\n",
    "        outputs, states = tf.nn.dynamic_rnn(cell, self.x, dtype=tf.float32)\n",
    "        num_examples = tf.shape(self.x)[0]\n",
    "        W_repeated = tf.tile(tf.expand_dims(self.W_out, 0), [num_examples, 1, 1])\n",
    "        out = tf.matmul(outputs, W_repeated) + self.b_out\n",
    "        out = tf.squeeze(out)\n",
    "        return out\n",
    "\n",
    "    def train(self, train_x, train_y):\n",
    "        with tf.Session() as sess:\n",
    "            tf.get_variable_scope().reuse_variables()\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "            for i in range(1000):\n",
    "                _, mse = sess.run([self.train_op, self.cost], feed_dict={self.x: train_x, self.y: train_y})\n",
    "                if i % 100 == 0:\n",
    "                    print(i, mse)\n",
    "            save_path = self.saver.save(sess, 'model.ckpt')\n",
    "            print('Model saved to {}'.format(save_path))\n",
    "\n",
    "    def test(self, test_x):\n",
    "        with tf.Session() as sess:\n",
    "            tf.get_variable_scope().reuse_variables()\n",
    "            self.saver.restore(sess, './model.ckpt')\n",
    "            output = sess.run(self.model(), feed_dict={self.x: test_x})\n",
    "            return output\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we'll train a series predictor. Let's say we have a sequence of numbers `[a, b, c, d]` that we want to transform into `[a, a+b, b+c, c+d]`. We'll give the RNN a couple examples in the training data. Let's see how well it learns this intended transformation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 103.46295\n",
      "100 63.418705\n",
      "200 23.072838\n",
      "300 11.47684\n",
      "400 7.195353\n",
      "500 4.4564924\n",
      "600 2.8910196\n",
      "700 1.948163\n",
      "800 1.3193887\n",
      "900 0.88628125\n",
      "Model saved to model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from ./model.ckpt\n",
      "\n",
      "Lets run some tests!\n",
      "\n",
      "When the input is [[1], [2], [3], [4]]\n",
      "The ground truth output should be [[1], [3], [5], [7]]\n",
      "And the model thinks it is [0.86705637 2.7930977  5.307706   7.302184  ]\n",
      "\n",
      "When the input is [[4], [5], [6], [7]]\n",
      "The ground truth output should be [[4], [9], [11], [13]]\n",
      "And the model thinks it is [ 4.0726233  9.083956  11.937489  12.943668 ]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "    predictor = SeriesPredictor(input_dim=1, seq_size=4, hidden_dim=10)\n",
    "    train_x = [[[1], [2], [5], [6]],\n",
    "               [[5], [7], [7], [8]],\n",
    "               [[3], [4], [5], [7]]]\n",
    "    train_y = [[1, 3, 7, 11],\n",
    "               [5, 12, 14, 15],\n",
    "               [3, 7, 9, 12]]\n",
    "    predictor.train(train_x, train_y)\n",
    "\n",
    "    test_x = [[[1], [2], [3], [4]],  # 1, 3, 5, 7\n",
    "              [[4], [5], [6], [7]]]  # 4, 9, 11, 13\n",
    "    actual_y = [[[1], [3], [5], [7]],\n",
    "                [[4], [9], [11], [13]]]\n",
    "    pred_y = predictor.test(test_x)\n",
    "    \n",
    "    print(\"\\nLets run some tests!\\n\")\n",
    "    \n",
    "    for i, x in enumerate(test_x):\n",
    "        print(\"When the input is {}\".format(x))\n",
    "        print(\"The ground truth output should be {}\".format(actual_y[i]))\n",
    "        print(\"And the model thinks it is {}\\n\".format(pred_y[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
